package com.taoqi.service.common;


import com.alibaba.fastjson.JSON;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.*;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CookieStore;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.*;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.cookie.CookieSpecProvider;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.*;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.impl.cookie.BestMatchSpecFactory;
import org.apache.http.impl.cookie.BrowserCompatSpecFactory;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.ExecutionContext;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.ssl.SSLContexts;
import org.apache.http.util.EntityUtils;
import org.jsoup.helper.*;
import org.jsoup.helper.HttpConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.net.ssl.SSLContext;
import java.io.*;
import java.net.*;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Map.Entry;

/**
 * Created by TQ on 2017/6/8.
 */
public class HttpClientUtils {

    private final static Logger LOGGER = LoggerFactory.getLogger(HttpClientUtils.class);

    private final static String UTF8 = "UTF-8";

    public static CloseableHttpClient httpclient = null;

    public static HttpClientBuilder builder = null;

    public static boolean isKeepLive = false;

    public static boolean isNeedProxy = false;

    public static boolean openDubug = false;

    public static ProxyReq currentProxy = null;

    public static List<ProxyReq> proxyReqs = null;

    public static String tempUrl = null;

    public static Map<String, Object> tempParameters = null;
    private static String keyWords = null;

    //请求 是否重试
    private Map<String, Integer> reqMap = new HashMap();

    static {
        // 初始化线程池
        RequestConfig params = RequestConfig.custom().setConnectTimeout(50000).setConnectionRequestTimeout(50000).setSocketTimeout(50000)
                .setExpectContinueEnabled(true).build();

        PoolingHttpClientConnectionManager pccm = new PoolingHttpClientConnectionManager();
        pccm.setMaxTotal(300); // 连接池最大并发连接数
        pccm.setDefaultMaxPerRoute(50); // 单路由最大并发数

        HttpRequestRetryHandler retryHandler = new HttpRequestRetryHandler() {
            public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
                // 重试1次,从1开始
                if (executionCount > 2) {
                    /*if (isNeedProxy) {
                        currentProxy = nextProxy(tempUrl, proxyReqs, true);
                        HttpHost proxy = new HttpHost(currentProxy.getProxyHost(), currentProxy.getProxyPort());
                        RequestConfig config = RequestConfig.custom()
                                .setProxy(proxy)
                                .build();

                        HttpClientContext clientContext = HttpClientContext.adapt(context);
                        HttpRequest request = clientContext.getRequest();
                        clientContext.setRequestConfig(config);
                        //boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
                        //post(tempUrl, tempParameters, 0, 0, 0);
                        LOGGER.info("切换代理IP!");
                    }*/
                    return true;
                }
                if (exception instanceof NoHttpResponseException) {
                    LOGGER.info(
                            "[NoHttpResponseException has retry request:" + context.toString() + "][executionCount:" + executionCount + "]");
                    return true;
                } else if (exception instanceof SocketException) {
                    LOGGER.info("[SocketException has retry request:" + context.toString() + "][executionCount:" + executionCount + "]");
                    return true;
                }
                return false;
            }
        };
        builder = HttpClients.custom().setConnectionManager(pccm).setDefaultRequestConfig(params).setRetryHandler(retryHandler);
        //httpclient = HttpClients.custom().setConnectionManager(pccm).setDefaultRequestConfig(params).setRetryHandler(retryHandler).build();
    }

    public HttpClientUtils build() {
        httpclient = builder.build();
        return this;
    }

    public static HttpClientUtils init() {
        return new HttpClientUtils();
    }

    public HttpClientUtils openDubug() {
        openDubug = true;
        return this;
    }

    public HttpClientUtils keepLive() {
        cookieStore = new BasicCookieStore();
        builder.setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy())
                .setRedirectStrategy(new DefaultRedirectStrategy())
                .setDefaultCookieStore(cookieStore);
        isKeepLive = true;
        return this;
    }

    public void close() {
        try {
            httpclient.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static String post(String urlToRequest, Map<String, Object> parameters, String dataType, Integer connectionRequestTimeout,
                              Integer connectTimeout, Integer socketTimeout, CloseableHttpClient client) {
        tempUrl = urlToRequest;
        tempParameters = parameters;
        HttpPost post = new HttpPost(urlToRequest);
        setHeaders(post);
        Long startTs = System.currentTimeMillis();
        List<NameValuePair> nvps = new ArrayList<NameValuePair>();
        if (StringUtils.isNotBlank(dataType)) {
            if ("json".equals(dataType)) {
                if (parameters != null && !parameters.isEmpty()) {
                    post.addHeader("Content-type", "application/json; charset=utf-8");
                    post.setHeader("Accept", "application/json");
                    post.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
                    post.setHeader("Connection", "keep-alive");
                    post.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko");
                    post.setHeader("Upgrade-insecure-Requests", "1");
                    post.setEntity(new StringEntity(JSON.toJSONString(parameters), Charset.forName("UTF-8")));
                }
            }
        } else {
            if (parameters != null && !parameters.isEmpty()) {
                for (Entry<String, Object> entry : parameters.entrySet()) {
                    nvps.add(new BasicNameValuePair(entry.getKey(), String.valueOf(entry.getValue())));
                }
                try {
                    post.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
                    post.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                    post.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
                    post.setHeader("Accept-Encoding", "gzip, deflate");
                    post.setHeader("Connection", "keep-alive");
                    post.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko");
                    //httpPost2.setHeader("Host", "www.court.gov.cn");
                    post.setHeader("Upgrade-insecure-Requests", "1");
                    post.setEntity(new UrlEncodedFormEntity(nvps, UTF8));
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }


        try {
            LOGGER.info("post-req:url:{},param:{}", urlToRequest, JSON.toJSONString(parameters));

            //添加代理
            if (isNeedProxy) {
                if (null == currentProxy) {
                    ProxyReq proxy = nextProxy(urlToRequest, proxyReqs, false);
                    if (null != proxy) {
                        addProxy(proxy.getProxyHost(), proxy.getProxyPort(), post);
                        LOGGER.info("当前使用代理地址: " + proxy.toString());
                    }
                } else {
                    addProxy(currentProxy.getProxyHost(), currentProxy.getProxyPort(), post);
                    LOGGER.info("当前使用代理地址: " + currentProxy.toString());
                }
            }

            /*if (connectionRequestTimeout != null && connectTimeout != null && socketTimeout != null) {
                RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(connectTimeout)
                        .setConnectionRequestTimeout(connectionRequestTimeout).setSocketTimeout(socketTimeout).build();
                post.setConfig(requestConfig);
            }*/

            String result = invoke(post, client);
            Long endTs = System.currentTimeMillis();
            Long currentMethodCallTime = endTs - startTs;
            if (currentMethodCallTime > 5000) {
                LOGGER.warn("url:{},call time {} ms", urlToRequest, currentMethodCallTime);
                LOGGER.info("所有存活线程=" + Thread.getAllStackTraces().size());
            }
            LOGGER.info("post-rps:{}", result);
            return result;
        } catch (Exception e) {
            LOGGER.error("[HttpClientUtils][post][Unsupported Encoding Exception]", e);
        }
        return null;
    }

    public static String get(String urlToRequest, Map<String, Object> parameters, String dataType, Integer connectionRequestTimeout,
                             Integer connectTimeout, Integer socketTimeout, CloseableHttpClient client) {
        tempUrl = urlToRequest;
        tempParameters = parameters;
        HttpGet get = new HttpGet(urlToRequest);
        Long startTs = System.currentTimeMillis();
        List<NameValuePair> nvps = new ArrayList<NameValuePair>();
        if (StringUtils.isNotBlank(dataType)) {
            if ("json".equals(dataType)) {
                if (parameters != null && !parameters.isEmpty()) {
                    get.addHeader("Content-type", "application/json; charset=utf-8");
                    get.setHeader("Accept", "application/json");
                    get.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
                    get.setHeader("Connection", "keep-alive");
                    get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko");
                    get.setHeader("Upgrade-insecure-Requests", "1");
                }
            }
        } else {
            if (parameters != null && !parameters.isEmpty()) {
                for (Entry<String, Object> entry : parameters.entrySet()) {
                    nvps.add(new BasicNameValuePair(entry.getKey(), String.valueOf(entry.getValue())));
                }
            }
            try {
                get.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
                get.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                get.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
                get.setHeader("Accept-Encoding", "gzip, deflate");
                get.setHeader("Connection", "keep-alive");
                get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko");
                //httpPost2.setHeader("Host", "www.court.gov.cn");
                get.setHeader("Upgrade-insecure-Requests", "1");
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        try {
            LOGGER.info("get-req:url:{},param:{}", urlToRequest, JSON.toJSONString(parameters));
            //添加代理
            if (isNeedProxy) {
                if (null == currentProxy) {
                    ProxyReq proxy = nextProxy(urlToRequest, proxyReqs, false);
                    if (null != proxy) {
                        addProxyGet(proxy.getProxyHost(), proxy.getProxyPort(), get);
                        LOGGER.info("当前使用代理地址: " + proxy.toString());
                    }
                } else {
                    addProxyGet(currentProxy.getProxyHost(), currentProxy.getProxyPort(), get);
                    LOGGER.info("当前使用代理地址: " + currentProxy.toString());
                }
            }

            /*if (connectionRequestTimeout != null && connectTimeout != null && socketTimeout != null) {
                RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(connectTimeout)
                        .setConnectionRequestTimeout(connectionRequestTimeout).setSocketTimeout(socketTimeout).build();
                post.setConfig(requestConfig);
            }*/

            String result = invoke(get, client);
            Long endTs = System.currentTimeMillis();
            Long currentMethodCallTime = endTs - startTs;
            if (currentMethodCallTime > 5000) {
                LOGGER.warn("url:{},call time {} ms", urlToRequest, currentMethodCallTime);
                LOGGER.info("所有存活线程=" + Thread.getAllStackTraces().size());
            }
            LOGGER.info("post-rps:{}", result);
            return result;
        } catch (Exception e) {
            LOGGER.error("[HttpClientUtils][post][Unsupported Encoding Exception]", e);
        }
        return null;
    }


    private static String invoke(HttpUriRequest request, CloseableHttpClient client) {
        CloseableHttpResponse response = null;
        try {
            response = client.execute(request);
            if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                HttpEntity entity = response.getEntity();
                String responseString = null;
                if (null != entity) {
                    responseString = EntityUtils.toString(entity, UTF8);
                }
                if (openDubug) {
                    // 响应状态
                    LOGGER.info("status:" + response.getStatusLine());
                    LOGGER.info("headers:");
                    HeaderIterator iterator = response.headerIterator();
                    while (iterator.hasNext()) {
                        LOGGER.info("\t" + iterator.next());
                    }
                    // 判断响应实体是否为空
                    if (entity != null) {
                        LOGGER.info("response length:" + responseString.length());
                        LOGGER.info("response content:"
                                + responseString.replace("\r\n", ""));
                    }
                }
                return responseString;
            } else {
                HttpEntity entity = response.getEntity();
                String responseString = null;
                if (null != entity) {
                    responseString = EntityUtils.toString(entity, UTF8);
                }
                String cookie = setCookie(response);

                if (openDubug) {
                    // 响应状态
                    LOGGER.info("status:" + response.getStatusLine());
                    LOGGER.info("headers:");
                    HeaderIterator iterator = response.headerIterator();
                    while (iterator.hasNext()) {
                        LOGGER.info("\t" + iterator.next());
                    }
                    // 判断响应实体是否为空
                    if (entity != null) {
                        LOGGER.info("response length:" + responseString.length());
                        LOGGER.info("response content:"
                                + responseString.replace("\r\n", ""));
                    }
                }
            }
        } catch (IOException e) {
            LOGGER.error(
                    "[HttpClientUtils][invoke][method:" + request.getMethod() + " URI:" + request.getURI() + "] is request exception", e);
        } finally {
            if (response != null) {
               /* try {
                    response.close();
                } catch (IOException e) {
                    LOGGER.error(
                            "[HttpClientUtils][invoke][method:" + request.getMethod() + " URI:" + request.getURI() + "] is closed exception",
                            e);
                }*/
            }
        }
        return null;
    }

    /**
     * 单个代理
     *
     * @param proxyIP
     * @param hostPort
     * @param user
     * @param pwd
     * @return
     */
    public HttpClientUtils setProxy(String proxyIP, int hostPort, String user, String pwd) {
        LOGGER.info("正在设置代理：" + proxyIP + ":" + hostPort);
        isNeedProxy = true;
        /*DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        builder.setRoutePlanner(routePlanner);*/
        CredentialsProvider credsProvider = new BasicCredentialsProvider();
        credsProvider.setCredentials(new AuthScope(proxyIP, hostPort), new UsernamePasswordCredentials(user, pwd));
        builder.setDefaultCredentialsProvider(credsProvider);
        ProxyReq proxyReq = new ProxyReq(proxyIP, hostPort, user, pwd);
        currentProxy = proxyReq;
        return this;
    }

    /**
     * 代理池
     *
     * @param proxys
     * @return
     */
    public HttpClientUtils setProxy(List<ProxyReq> proxys) {
        isNeedProxy = true;
        if (null != proxys && proxys.size() > 0) {
            CredentialsProvider credsProvider = new BasicCredentialsProvider();
            Vector<ProxyReq> vv = new Vector();
            vv.addAll(proxys);
            Iterator<ProxyReq> reqIt = vv.iterator();
            while (reqIt.hasNext()) {
                ProxyReq proxy = reqIt.next();
                credsProvider.setCredentials(new AuthScope(proxy.getProxyHost(), proxy.getProxyPort()),
                        new UsernamePasswordCredentials(proxy.getProxyUser(), proxy.getProxyPwd()));
            }
            proxyReqs = proxys;
            builder.setDefaultCredentialsProvider(credsProvider);
        }
        return this;
    }

    /**
     * 测试代理服务器的可用性
     * 只有返回HttpStatus.SC_OK才说明代理服务器有效
     * 其他的都是不行的
     */
    private static int testProxyServer(String url, String proxyIp, int proxyPort, String user, String pwd, CloseableHttpClient client) {
        ProxyReq proxyReq = new ProxyReq(proxyIp, proxyPort, user, pwd);
        currentProxy = proxyReq;
        HttpPost postMethod = new HttpPost(url);
        addProxy(proxyIp, proxyPort, postMethod);
        if (postMethod == null) {
            System.out.println("请求协议设置都搞错了，所以我无法完成您的请求");
            return 0;
        }
        try {
            CloseableHttpResponse response = client.execute(postMethod);
            if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { //2XX状态码
                return 1;
            } else if (response.getStatusLine().getStatusCode() == HttpStatus.SC_FORBIDDEN) { //代理还是不行
                return 0;
            } else {  //  其他的错误
                return 0;
            }
        } catch (IOException e) {
            // 发生网络异常
            //System.exit(1);
        } finally {
            // 释放连接
            postMethod.releaseConnection();
        }
        return 0;
    }

    public static ProxyReq nextProxy(String url, List<ProxyReq> proxys, boolean isRetry) {
        if (null != proxys && proxys.size() > 0) {
            Iterator<ProxyReq> it = proxys.iterator();
            while (it.hasNext()) {
                ProxyReq req = it.next();
                if (isRetry) {
                    if (req.getProxyHost().equals(currentProxy.getProxyHost()) && req.getProxyPort() == currentProxy.getProxyPort()) {
                        it.remove();
                        LOGGER.info("删除不给力的节点: " + currentProxy.toString());
                        isRetry = false;
                        continue;
                    }
                }
                if (TelnetUtils.checkProxyHost(req.getProxyHost(), req.getProxyPort())) {
                    int res = testProxyServer(url, req.getProxyHost(), req.getProxyPort(), req.getProxyUser(), req.getProxyPwd(), httpclient);
                    if (1 != res) {
                        it.remove();
                    } else {
                        return req;
                    }
                } else {
                    it.remove();
                    LOGGER.info("删除不给力的节点: " + req.toString());
                }
            }
        } else {
            LOGGER.info("代理池已为空!");
            List<ProxyReq> newProxys = ProxyIpFetcher.fetchAll(url,new String[]{keyWords});
            LOGGER.info("获取到新的代理IP个数: " +  newProxys.size() + "个");
            //重新获取一批代理
            proxyReqs =  newProxys;
            //System.exit(0);
        }
        if (null != proxyReqs && proxyReqs.size() == 0) {
            List<ProxyReq> newProxys = ProxyIpFetcher.fetchAll("http://blog.csdn.net/whos2002110/article/details/42773081",new String[]{"爬虫"});
            LOGGER.info("获取到新的代理IP个数: " +  newProxys.size() + "个");
            //重新获取一批代理
            proxyReqs =  newProxys;
            return nextProxy(url,newProxys, true);
        }
        return null;
    }


    static CookieStore cookieStore = null;
    static HttpClientContext context = null;

    public static void setContext() {
        System.out.println("----setContext");
        context = HttpClientContext.create();
        Registry<CookieSpecProvider> registry = RegistryBuilder
                .<CookieSpecProvider>create()
                .register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())
                .register(CookieSpecs.BROWSER_COMPATIBILITY,
                        new BrowserCompatSpecFactory()).build();
        context.setCookieSpecRegistry(registry);
        context.setCookieStore(cookieStore);
    }

    public static Map<String, String> cookieMap = new HashMap<String, String>(64);

    public static String setCookie(HttpResponse httpResponse) {
        System.out.println("----setCookieStore");
        Header headers[] = httpResponse.getHeaders("Set-Cookie");
        if (headers == null || headers.length == 0) {
            System.out.println("----there are no cookies");
            return null;
        }
        String cookie = "";
        for (int i = 0; i < headers.length; i++) {
            cookie += headers[i].getValue();
            if (i != headers.length - 1) {
                cookie += ";";
            }
        }

        String cookies[] = cookie.split(";");
        for (String c : cookies) {
            c = c.trim();
            if (cookieMap.containsKey(c.split("=")[0])) {
                cookieMap.remove(c.split("=")[0]);
            }
            cookieMap.put(c.split("=")[0], c.split("=").length == 1 ? "" : (c.split("=").length == 2 ? c.split("=")[1] : c.split("=", 2)[1]));
        }
        System.out.println("----setCookieStore success");
        String cookiesTmp = "";
        for (String key : cookieMap.keySet()) {
            cookiesTmp += key + "=" + cookieMap.get(key) + ";";
        }

        return cookiesTmp.substring(0, cookiesTmp.length() - 2);
    }

    public static List<NameValuePair> getParam(Map parameterMap) {
        List<NameValuePair> param = new ArrayList<NameValuePair>();
        Iterator it = parameterMap.entrySet().iterator();
        while (it.hasNext()) {
            Entry parmEntry = (Entry) it.next();
            param.add(new BasicNameValuePair((String) parmEntry.getKey(),
                    (String) parmEntry.getValue()));
        }
        return param;
    }

    public static void printResponse(HttpResponse httpResponse)
            throws ParseException, IOException {
        // 获取响应消息实体
        HttpEntity entity = httpResponse.getEntity();
        // 响应状态
        System.out.println("status:" + httpResponse.getStatusLine());
        System.out.println("headers:");
        HeaderIterator iterator = httpResponse.headerIterator();
        while (iterator.hasNext()) {
            System.out.println("\t" + iterator.next());
        }
        // 判断响应实体是否为空
        if (entity != null) {
            String responseString = EntityUtils.toString(entity);
            System.out.println("response length:" + responseString.length());
            System.out.println("response content:"
                    + responseString.replace("\r\n", ""));
        }
    }


    public static void addProxy(String proxyIP, int port, HttpPost httpPost) {
        HttpHost proxy = new HttpHost(proxyIP, port);
        RequestConfig config = RequestConfig.custom()
                .setProxy(proxy)
                .build();
        httpPost.setConfig(config);
    }

    public static void addProxyGet(String proxyIP, int port, HttpGet httpGet) {
        HttpHost proxy = new HttpHost(proxyIP, port);
        RequestConfig config = RequestConfig.custom()
                .setProxy(proxy)
                .build();
        httpGet.setConfig(config);
    }

    //-------------------------------并发请求        start
    public HttpClientUtils duang(List<String> urls) {
        //CloseableHttpClient httpclient = build();
        try {
            String[] urisToGet = urls.toArray(new String[urls.size()]);
            GetThread[] threads = new GetThread[urisToGet.length];
            for (int i = 0; i < threads.length; i++) {
                HttpGet httpget = new HttpGet(urisToGet[i]);
                threads[i] = new GetThread(httpclient, httpget, i + 1);
            }

            for (int j = 0; j < threads.length; j++) {
                threads[j].start();
            }

            for (int j = 0; j < threads.length; j++) {
                threads[j].join();
            }

        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                if (!isKeepLive) {
                    httpclient.close();
                }
            } catch (Exception e) {

            }
        }
        return this;
    }

    GetThread gt = new GetThread();

    static class GetThread extends Thread implements HttpClientUtils.DataDual {

        private  CloseableHttpClient httpClient = null;
        private  HttpContext context = null;
        private  HttpGet httpget = null;
        private  int id = 0;

        public GetThread() {

        }

        public int process(String result) {
            return 0;
        }

        public GetThread(CloseableHttpClient httpClient, HttpGet httpget, int id) {
            this.httpClient = httpClient;
            this.context = new BasicHttpContext();
            this.httpget = httpget;
            this.id = id;
        }

        public void run() {
            try {
                System.out.println(id + " - about to get something from " + httpget.getURI());
                CloseableHttpResponse response = httpClient.execute(httpget, context);
                try {
                    System.out.println(id + " - get executed");
                    // get the response body as an array of bytes
                    HttpEntity entity = response.getEntity();
                    /*if (entity != null) {
                        byte[] bytes = EntityUtils.toByteArray(entity);
                        System.out.println(id + " - " + bytes.length + " bytes read");
                    }*/
                    String responseString = EntityUtils.toString(entity, UTF8);

                } finally {
                    response.close();
                }
            } catch (Exception e) {
                System.out.println(id + " - error: " + e);
            }
        }
    }

    public interface DataDual {
        int process(String result);
    }

    //-------------------------------并发请求        end

    //-------------------------------http隧道 适用与https代理         start
    public static void proxyTennel() {
        ProxyClient proxyClient = new ProxyClient();
        HttpHost target = new HttpHost("www.baidu.com", 80);
        HttpHost proxy = new HttpHost("localhost", 8888);
        UsernamePasswordCredentials credentials = new UsernamePasswordCredentials("user", "pwd");
        Socket socket = null;
        try {
            socket = proxyClient.tunnel(proxy, target, credentials);
            Writer out = new OutputStreamWriter(socket.getOutputStream(), HTTP.DEF_CONTENT_CHARSET);
            out.write("GET / HTTP/1.1\r\n");
            out.write("Host: " + target.toHostString() + "\r\n");
            out.write("Agent: whatever\r\n");
            out.write("Connection: close\r\n");
            out.write("\r\n");
            out.flush();
            BufferedReader in = new BufferedReader(
                    new InputStreamReader(socket.getInputStream(), HTTP.DEF_CONTENT_CHARSET));
            String line = null;
            while ((line = in.readLine()) != null) {
                System.out.println(line);
            }
        } catch (TunnelRefusedException e) {
            e.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                socket.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    //-------------------------------http隧道 试用与https代理         end

    public void sslConn() {
        SSLContext sslcontext = null;
        try {
            sslcontext = SSLContexts.custom()
                    .loadTrustMaterial(new File("D:\\what21.jks"),
                            "what21".toCharArray(),
                            new TrustSelfSignedStrategy())
                    .build();
        } catch (Exception e) {
        }
        // 只允许TLSv1协议
        SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(
                sslcontext,
                new String[]{"TLSv1"},
                null,
                SSLConnectionSocketFactory.getDefaultHostnameVerifier());
        // 创建HTTPClient
        CloseableHttpClient httpclient = HttpClients.custom()
                // SSL Factory
                .setSSLSocketFactory(sslsf)
                .build();
        try {
            HttpGet httpget = new HttpGet("https://www.what21.com/");
            System.out.println("执行请求： " + httpget.getRequestLine());
            CloseableHttpResponse response = httpclient.execute(httpget);
            try {
                HttpEntity entity = response.getEntity();
                System.out.println("----------------------------------------");
                System.out.println("返回响应: " + response.getStatusLine());
                System.out.println("响应内容：" + EntityUtils.toString(entity));
                System.out.println("----------------------------------------");

            } finally {
                response.close();
            }
        } catch (Exception e) {
        } finally {
            try {
                httpclient.close();
            } catch (Exception e) {
            }
        }
    }

    public static String[] downFileCode(String url, String destFileName, CloseableHttpClient httpclient) {
        String[] result = new String[3];
        InputStream in = null;
        String files = "";
        FileOutputStream fos = null;
        FileOutputStream fileOutputStream = null;
        try {
            HttpContext httpContext = new BasicHttpContext();
            HttpGet httpGet = new HttpGet(url);
//            httpGet.setHeader("Accept", "text/html, application/xhtml+xml, image/jxr, */*");
//            httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko");
//            httpGet.setHeader("Connection", "Keep-Alive");
//            //httpGet.setHeader("host", "www.court.gov.cn");
//            httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
//            httpGet.setHeader("Upgrade-Insecure-Requests", "1");
//            httpGet.setHeader("Accept-Encoding", "gzip, deflate");
            HttpResponse response = httpclient.execute(httpGet, httpContext);
            HttpHost targetHost = (HttpHost) httpContext.getAttribute(ExecutionContext.HTTP_TARGET_HOST);
            SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
            Date date = new Date();
            files = destFileName + sdf.format(date) + "_" + System.currentTimeMillis() + ".jpeg";
            int code = response.getStatusLine().getStatusCode();
            LOGGER.info("下载验证码：" + code);
            result[0] = code + "";
            result[1] = files;
            result[2] = setCookie(response);
            fileOutputStream = new FileOutputStream(new File(files));
            response.getEntity().writeTo(fileOutputStream);
            fileOutputStream.flush();
            if (openDubug) {
                // 响应状态
                System.out.println("status:" + response.getStatusLine());
                System.out.println("headers:");
                HeaderIterator iterator = response.headerIterator();
                while (iterator.hasNext()) {
                    System.out.println("\t" + iterator.next());
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                if (null != fos) {
                    fos.close();
                }
                if (null != in) {
                    in.close();
                }
                if (null != fileOutputStream) {
                    fileOutputStream.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
            if (!isKeepLive) {
                try {
                    httpclient.close();
                } catch (IOException e) {
                }
            }
        }
        return result;
    }

    public static String[] downFile(String url, String destFileName, CloseableHttpClient httpclient) {
        String[] result = new String[2];
        String[] fc = downFileCode(url, destFileName, httpclient);
        int retryCount = 0;
        int retrySum = 5;
        while (retryCount < retrySum)
            if (null != fc && fc.length == 3) {
                if (Integer.valueOf(fc[0]) == 200) {
                    result[0] = fc[1];
                    result[1] = fc[2];
                    retryCount = retrySum;
                } else {
                    fc = downFileCode(url, destFileName, httpclient);
                    retryCount++;
                }
            }
        return result;
    }

    public static void setHeaders(HttpEntityEnclosingRequestBase post) {
        post.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36");
        post.setHeader("Accept-Encoding", "gzip");
        post.setHeader("charset", "UTF-8");
    }

    public static void download3(String reqUrl, String remoteFileName, String localFileName) {
        FileOutputStream out = null;
        InputStream in = null;

        try {
            URL url = new URL(reqUrl);
            URLConnection urlConnection = url.openConnection();
            HttpURLConnection httpURLConnection = (HttpURLConnection) urlConnection;

            // true -- will setting parameters
            httpURLConnection.setDoOutput(true);
            // true--will allow read in from
            httpURLConnection.setDoInput(true);
            // will not use caches
            httpURLConnection.setUseCaches(false);
            // setting serialized
            httpURLConnection.setRequestProperty("Content-type", "application/x-java-serialized-object");
            // default is GET
            httpURLConnection.setRequestMethod("POST");
            httpURLConnection.setRequestProperty("connection", "Keep-Alive");
            httpURLConnection.setRequestProperty("Charsert", "UTF-8");
            // 1 min
            httpURLConnection.setConnectTimeout(60000);
            // 1 min
            httpURLConnection.setReadTimeout(60000);

           /* httpURLConnection.addRequestProperty("userName", userName);
            httpURLConnection.addRequestProperty("passwd", passwd);*/
            httpURLConnection.addRequestProperty("fileName", remoteFileName);

            // connect to server (tcp)
            httpURLConnection.connect();

            in = httpURLConnection.getInputStream();// send request to
            // server
            File file = new File(localFileName);
            if (!file.exists()) {
                file.createNewFile();
            }

            out = new FileOutputStream(file);
            byte[] buffer = new byte[4096];
            int readLength = 0;
            while ((readLength = in.read(buffer)) > 0) {
                byte[] bytes = new byte[readLength];
                System.arraycopy(buffer, 0, bytes, 0, readLength);
                out.write(bytes);
            }

            out.flush();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                if (in != null) {
                    in.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }

            try {
                if (out != null) {
                    out.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }


    /**
     * @param hasVerifcaCode 如果需要验证码则需要添加验证码url地址
     * @return
     */
    public static boolean login(Map<String, Object> attr, boolean hasVerifcaCode, CloseableHttpClient client) {
        boolean isLogin = false;
        HttpGet mainPage = null;
        if (attr.containsKey(Constants.MAIN_PAGE_URL)) {
            mainPage = new HttpGet(attr.get(Constants.MAIN_PAGE_URL).toString());
            attr.remove(Constants.MAIN_PAGE_URL);
        }
        HttpPost loginPost = null;
        if (attr.containsKey(Constants.MAIN_LOGIN_URL)) {
            loginPost = new HttpPost(attr.get(Constants.MAIN_LOGIN_URL).toString());
            attr.remove(Constants.MAIN_LOGIN_URL);
        }
        try {
            client.execute(mainPage);
            if (hasVerifcaCode) {
                String[] file = null;
                if (attr.containsKey(Constants.VERIFY_CODE_URL)) {
                    String codeUrl = attr.get(Constants.VERIFY_CODE_URL).toString();
                    if (StringUtils.isNotBlank(codeUrl)) {
                        file = downFile(codeUrl, "D:/yzm/", client);
                        LOGGER.info("验证码文件: " + file[0]);
                    }
                }
                System.out.println("verifying code has been save as verifyCode.jpeg, input its content");
                String code;
                Scanner in = new Scanner(System.in);
                code = in.nextLine();
                in.close();
                if (attr.containsKey(Constants.VERIFY_CODE_ATTR_NAME)) {
                    String attrName = attr.get(Constants.VERIFY_CODE_ATTR_NAME).toString();
                    if (StringUtils.isNotBlank(attrName)) {
                        attr.put(attrName, code);
                    }
                }
                System.out.println("cookie: " + file[1]);
            }
            attr.remove(Constants.VERIFY_CODE_URL);
            attr.remove(Constants.VERIFY_CODE_ATTR_NAME);
            // 执行post请求
            //httpPost2.setHeader("Cookie", file[1]);//将get得到的cookie放进去
            loginPost.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
            loginPost.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
            loginPost.setHeader("Accept-Encoding", "gzip, deflate");
            loginPost.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
            loginPost.setHeader("Connection", "keep-alive");
            loginPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko");
            //httpPost2.setHeader("Host", "www.court.gov.cn");
            loginPost.setHeader("Upgrade-insecure-Requests", "1");
            List<NameValuePair> postData = new ArrayList();
            if (attr != null && !attr.isEmpty()) {
                for (Entry<String, Object> entry : attr.entrySet()) {
                    postData.add(new BasicNameValuePair(entry.getKey(), String.valueOf(entry.getValue())));
                }
            }
            loginPost.setEntity(new UrlEncodedFormEntity(postData, Consts.UTF_8));
            //执行post得到结果
            HttpResponse httpResponse1 = client.execute(loginPost);
            printResponse(httpResponse1);
            System.out.println("result" + httpResponse1);
            // context
            setContext();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return isLogin;
    }


    public static void main(String[] args) {

        //proxyTennel();
        //验证码下载
        //downFile("https://ss1.baidu.com/6ONXsjip0QIZ8tyhnq/it/u=2434560235,935681266&fm=58", "d:/eclipse.jpg");

        System.out.println("----testLogin");

        String proxyIP = "60.169.5.68";
        int port = 8080;
        List<ProxyReq> reqs = new ArrayList();
        reqs.add(new ProxyReq(proxyIP, port, "", ""));
        reqs.add(new ProxyReq("111.23.10.171", 80, "", ""));
        reqs.add(new ProxyReq("60.169.5.68", 8080, "", ""));
        reqs.add(new ProxyReq("120.198.244.50", 8080, "", ""));

        //CloseableHttpClient client = HttpClientUtils.init().setProxy(proxyIP, port, "", "").build();
        CloseableHttpClient client = null;// HttpClientUtils.init().setProxy(reqs).build();


        //testProxyServer("http://www.jokeji.cn/", proxyIP, port, "", "", client);
        //client = setProxy("111.75.235.37", 8998, "", "", client);
        HttpPost httpPost = new HttpPost("http://www.jokeji.cn/user/c.asp?u=zhangsan998&p=zhangsan998&sn=1&t=big");

        HttpClientUtils.addProxy(proxyIP, port, httpPost);
        Map parameterMap = new HashMap();
        try {
            UrlEncodedFormEntity postEntity = new UrlEncodedFormEntity(
                    getParam(parameterMap), "UTF-8");
            //httpPost.setEntity(postEntity);
            System.out.println("request line:" + httpPost.getRequestLine());
            // 执行post请求
            HttpResponse httpResponse = client.execute(httpPost);

            /*String location = httpResponse.getFirstHeader("Location").getValue();
            if (location != null && location.startsWith("http://www.baidu.com")) {
                System.out.println("----loginError");
            }*/
            printResponse(httpResponse);

            // 执行get请求
            System.out.println("----the same client");
            HttpGet httpGet = new HttpGet("http://www.jokeji.cn/User/MemberCenter.asp");
            //httpGet.setConfig(config);
            System.out.println("request line:" + httpGet.getRequestLine());
            HttpResponse httpResponse1 = client.execute(httpGet);
            printResponse(httpResponse1);
            // cookie store
            setCookie(httpResponse);
            // context
            setContext();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                // 关闭流并释放资源
                client.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}